package crawler.web.deduplicate;

import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.wltea.analyzer.lucene.IKAnalyzer;

public class IKUtil {
	/**
	 * 分词
	 * 
	 * @author waynechen
	 * @param text
	 * @return
	 * @throws IOException
	 */
	public static List<String> getWords(String text) throws IOException {
		List<String> list = new LinkedList<String>();
		IKAnalyzer analyzer = new IKAnalyzer();
		TokenStream tokenStream = analyzer.tokenStream("content", new StringReader(text));
		while (tokenStream.incrementToken()) {
			TermAttribute termAttribute = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
			list.add(termAttribute.term());
		}
		return list;
	}

	/**
	 * 获得前k个词, 根据其长度与频率<br>
	 * 长度一样的按照频率
	 * 
	 * @author waynechen
	 * @param text
	 * @param k
	 * @return
	 * @throws IOException
	 */
	public static List<String> getFirst$kWords(List<String> words, String text, int k) throws IOException {
		class WF {
			public String word;
			public int count;
		}
		List<WF> list = new ArrayList<WF>(words.size());
		for (String w : words) {
			WF wf = new WF();
			wf.word = w;
			wf.count = getNumInText(wf.word, text);
			list.add(wf);
		}
		Collections.sort(list, new Comparator<WF>() {
			@Override
			public int compare(WF o1, WF o2) {
				int i = o2.word.length() - o1.word.length();
				if (i != 0)
					return i;
				return o2.count - o1.count;
			}
		});
		List<String> ret = new ArrayList<String>(k);
		int i = 0;
		for (WF wf : list) {
			if (i++ >= k)
				break;
			ret.add(wf.word);
		}
		return ret;
	}

	private static int getNumInText(String key, String text) {
		if (key.length() <= 0)
			return 0;
		int count = 0;
		int index = 0;
		while (-1 != (index = text.indexOf(key, index))) {
			count++;
			index += key.length();
			if (index >= text.length())
				break;
		}
		return count;
	}
}
//
//
//
//
//
//
//
//
//
//
//
